Data Exploration

use curatedMetagenomicData.tsv to explore data

library(tidyverse)
library(readxl)
df <- read_tsv("raw-data/curatedMetagenomicData.tsv")
Rows: 20283 Columns: 130
── Column specification ─────────────────────────────────────────────────────────────────────
Delimiter: "\t"
chr (57): study_name, sample_id, subject_id, body_site, antibiotics_current_use, study_co...
dbl (73): age, infant_age, PMID, number_reads, number_bases, minimum_read_length, median_...

ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
df %>%
  group_by(study_name,study_condition) %>%
  summarise(count = n())
`summarise()` has grouped output by 'study_name'. You can override using the `.groups` argument.

find studies that included an internal control

df1 <- df %>% 
  group_by(study_name,study_condition) %>%
  summarise(count = n()) %>%
  select(study_name) %>% as.list()
vec <- df1$study_name
studyNamesWithCtr <- vec[ave(seq_along(vec), vec, FUN = length)>1] %>% unique()
studyNamesWithCtr
 [1] "BrooksB_2017"           "Castro-NallarE_2015"    "ChngKR_2016"           
 [4] "DavidLA_2015"           "FengQ_2015"             "GhensiP_2019"          
 [7] "GuptaA_2019"            "HallAB_2017"            "HanniganGD_2017"       
[10] "Heitz-BuschartA_2016"   "HMP_2019_ibdmdb"        "HMP_2019_t2d"          
[13] "IjazUZ_2017"            "JieZ_2017"              "KarlssonFH_2013"       
[16] "KieserS_2018"           "KosticAD_2015"          "LiJ_2014"              
[19] "LiJ_2017"               "LiSS_2016"              "LoombaR_2017"          
[22] "NagySzakalD_2017"       "NielsenHB_2014"         "QinJ_2012"             
[25] "QinN_2014"              "RaymondF_2016"          "RosaBA_2018"           
[28] "RubelMA_2020"           "SankaranarayananK_2015" "ShiB_2015"             
[31] "TettAJ_2016"            "ThomasAM_2018a"         "ThomasAM_2018b"        
[34] "ThomasAM_2019_c"        "VatanenT_2016"          "VincentC_2016"         
[37] "VogtmannE_2016"         "WirbelJ_2018"           "XieH_2016"             
[40] "YachidaS_2019"          "YeZ_2018"               "YuJ_2015"              
[43] "ZellerG_2014"           "ZhuF_2020"             

now that I have the list of studies with controls, I’ll check the first to make sure my search is accurate

df %>%
  group_by(study_name,study_condition) %>%
  summarise(count = n()) %>%
  filter(study_name == studyNamesWithCtr[1])

the list seems to have worked! Now find CRC studies that include internal Ctrs

studiesCRCwithCtr <- df %>%
  group_by(study_name,study_condition) %>%
  summarise(count = n()) %>%
  filter(study_name %in% studyNamesWithCtr[1:length(studyNamesWithCtr)]) %>%
  filter(study_condition == "CRC")
studiesCRCwithCtr

Save table

now that I have a table with only CRC studies with included Ctrs, I’ll look at their Metadata

studiesCRCwithCtr$study_name[1]
[1] "FengQ_2015"
df %>%
  filter(study_name == studiesCRCwithCtr$study_name[1])

FengQ_2015

This study is the first study from list with CRC and internal Ctrs. Although there is no annotation in the curatedMetagenomic table, the original article seem to have some I see if I can pull the table and merge. Below is a bunch of code that clean up Clinical table downloaed from original manuscript

# I confirmed there is lots of data in the published dataset
# curate the table

mdFengQ_2015_SupplData1_1 <- read_excel("raw-data/FengQ_2015_PMID25758642/41467_2015_BFncomms7528_MOESM1193_ESM.xlsx",
                                        skip = 2)
                                        
newHeader <- str_replace(string = names(mdFengQ_2015_SupplData1_1),pattern = "\\.\\.\\..+",replacement = "")

for (i in 1:length(newHeader)){
  hold <- newHeader[i]
  if (newHeader[i+1] == ""){
    newHeader[i+1] <- hold
  }
  # break check
  if (i==51) break
}

names(mdFengQ_2015_SupplData1_1) <- paste0(newHeader,"_",mdFengQ_2015_SupplData1_1[1,])
mdFengQ_2015_SupplData1_2 <- mdFengQ_2015_SupplData1_1[-1,]

names(mdFengQ_2015_SupplData1_2)[48] <- "MLG classifier_Type_Probability of carcinoma"
names(mdFengQ_2015_SupplData1_2)[50] <- "MLG classifier_Type_Probability of advanced adenoma"
names(mdFengQ_2015_SupplData1_2)[52] <- "MLG classifier_Type_Probability of advanced adenoma add age"
names(mdFengQ_2015_SupplData1_2)[1] <- "Sample ID"

names(mdFengQ_2015_SupplData1_2) <- str_replace_all(string = names(mdFengQ_2015_SupplData1_2),pattern = "[[:space:]]|\\(|\\/|\\)|\\:",replacement = "_")

mdFengQ_2015_SupplData1_3 <- mdFengQ_2015_SupplData1_2 %>%
  mutate(across(names(mdFengQ_2015_SupplData1_2[,3:25]), as.integer)) %>%
  mutate(across(names(mdFengQ_2015_SupplData1_2[,26:31]), as.factor)) %>%
  mutate(across(names(mdFengQ_2015_SupplData1_2[,32:38]), as.integer)) %>%
  mutate(across(names(mdFengQ_2015_SupplData1_2[,39:46]), as.factor)) %>%
  mutate(across(names(mdFengQ_2015_SupplData1_2[,47]), as.numeric)) %>%
  mutate(across(names(mdFengQ_2015_SupplData1_2[,48]), as.factor)) %>%
  mutate(across(names(mdFengQ_2015_SupplData1_2[,49]), as.numeric)) %>%
  mutate(across(names(mdFengQ_2015_SupplData1_2[,40]), as.factor))

mdFengQ_2015_SupplData1_3 <- mdFengQ_2015_SupplData1_3 %>%
  mutate(Clinical_data_State = factor(Clinical_data_State, levels=c("controls", "advanced adenoma","carcinoma")))

#edit tables as some muberic are actually factors
mdFengQ_2015_SupplData1_4 <- mdFengQ_2015_SupplData1_3 %>%
  mutate(across(names(mdFengQ_2015_SupplData1_3[,7:9]), as.factor)) %>%
  mutate(across(names(mdFengQ_2015_SupplData1_3[,16:17]), as.factor)) %>%
  mutate(across(names(mdFengQ_2015_SupplData1_3[,24:37]), as.factor)) %>%
  mutate(across(names(mdFengQ_2015_SupplData1_3[,39:46]), as.factor)) %>%
  mutate(across(names(mdFengQ_2015_SupplData1_3[,48]), as.factor)) %>%
  mutate(across(names(mdFengQ_2015_SupplData1_3[,50]), as.factor))


# I have now a well formatted table

mdFengQ_2015_SupplData1_4

I can now vidualize some data for exploration purpose


# Age across Status
mdFengQ_2015_SupplData1_3 %>%
  mutate(Clinical_data_State = factor(Clinical_data_State, levels=c("controls", "advanced adenoma","carcinoma"))) %>%
  
  ggplot(aes(x = Clinical_data_State,y = `Clinical_data_Age_(yrs)`)) +
  geom_violin(trim = T,na.rm = T) +
  geom_jitter(width = 0.2,size = 1) +
  
  theme_classic()


# BMI across Status
mdFengQ_2015_SupplData1_3 %>%
  mutate(Clinical_data_State = factor(Clinical_data_State, levels=c("controls", "advanced adenoma","carcinoma"))) %>%
  
  ggplot(aes(x = Clinical_data_State,y = Clinical_data_BMI)) +
  geom_violin(trim = T,na.rm = T) +
  geom_jitter(width = 0.2,size = 1) +
  
  theme_classic()


# Sex across Status
mdFengQ_2015_SupplData1_3 %>%
  mutate(Clinical_data_State = factor(Clinical_data_State, levels=c("controls", "advanced adenoma","carcinoma"))) %>%
  group_by(Clinical_data_State,`Clinical_data_Gender_(1:male,_2:female)`) %>%
  summarise(count = n())

# GGT across Status
mdFengQ_2015_SupplData1_3 %>%
  mutate(Clinical_data_State = factor(Clinical_data_State, levels=c("controls", "advanced adenoma","carcinoma"))) %>%
  
  ggplot(aes(x = Clinical_data_State,y = `Clinical_data_GGT_(U/L)`)) +
  geom_violin(trim = T,na.rm = T) +
  geom_jitter(width = 0.2,size = 1) +
  
  theme_classic()

Faster Correlation Visualization

Part of the future work will include the identificationof correlative values between certain Taxa or BGC and metadata. To speed up representation, I divide the variables (columns) between continous and factorial, and adopt differnet plotting strategy to identify linear regression

# Loop to print multiple exploratory Plots
## classify variables are numeric or factorial

numericVar <- mdFengQ_2015_SupplData1_4 %>%
  select_if(.predicate = is.numeric) %>% 
  names()

factorialVar <- mdFengQ_2015_SupplData1_4 %>%
  select_if(.predicate = is.factor) %>% 
  names()

#give name to each value in the vectors
numericVar = set_names(numericVar)
factorialVar = set_names(factorialVar)

# make plotting functions
## for continuous data, use scatter
scatter_fun = function(data,x,y){
  ggplot(data, aes(x = .data[[x]], y = .data[[y]])) +
    geom_point(na.rm=TRUE) +
    geom_smooth(method='lm',na.rm=TRUE) +
    theme_bw() +
    theme(axis.text = element_blank(),axis.title = element_text(size = 5))
}

## for factorial data, use violin
violin_fun = function(data,x, y) {
  ggplot(data, aes(x = .data[[x]], y = .data[[y]]) ) +
    geom_violin() +
    geom_jitter(size = 1, alpha = 0.5, width = 0.2) +
    theme_bw()
}

numericVar
                                       Sample_ID 
                                     "Sample_ID" 
                         Clinical_data_Age__yrs_ 
                       "Clinical_data_Age__yrs_" 
                               Clinical_data_BMI 
                             "Clinical_data_BMI" 
                        Clinical_data_Waist__cm_ 
                      "Clinical_data_Waist__cm_" 
                          Clinical_data_Hip__cm_ 
                        "Clinical_data_Hip__cm_" 
                         Clinical_data_GGT__U_L_ 
                       "Clinical_data_GGT__U_L_" 
                   Clinical_data_GOT__AST___U_L_ 
                 "Clinical_data_GOT__AST___U_L_" 
                   Clinical_data_GPT__ALT___U_L_ 
                 "Clinical_data_GPT__ALT___U_L_" 
           Clinical_data_Fasting_insulin__µU_mL_ 
         "Clinical_data_Fasting_insulin__µU_mL_" 
            Clinical_data_Fasting_glucose__mg_L_ 
          "Clinical_data_Fasting_glucose__mg_L_" 
                        Clinical_data_HOMA_index 
                      "Clinical_data_HOMA_index" 
                        Clinical_data_CRP__mg_L_ 
                      "Clinical_data_CRP__mg_L_" 
                  Clinical_data_Ferritin__ng_mL_ 
                "Clinical_data_Ferritin__ng_mL_" 
                          Clinical_data_Hb__g_L_ 
                        "Clinical_data_Hb__g_L_" 
                         Clinical_data_TG__mg_L_ 
                       "Clinical_data_TG__mg_L_" 
                        Clinical_data_HDL__mg_L_ 
                      "Clinical_data_HDL__mg_L_" 
                        Clinical_data_LDL__mg_L_ 
                      "Clinical_data_LDL__mg_L_" 
                             Fiber_intake_g_week 
                           "Fiber_intake_g_week" 
         MLG_classifier_Probability_of_carcinoma 
       "MLG_classifier_Probability_of_carcinoma" 
  MLG_classifier_Probability_of_advanced_adenoma 
"MLG_classifier_Probability_of_advanced_adenoma" 
factorialVar
                                                                 Clinical_data_State 
                                                               "Clinical_data_State" 
                                                 Clinical_data_WHR__waist-hip-ratio_ 
                                               "Clinical_data_WHR__waist-hip-ratio_" 
                                             Clinical_data_Gender__1_male,_2_female_ 
                                           "Clinical_data_Gender__1_male,_2_female_" 
                                Clinical_data_Fatty_liver_in_ultrasound_1_yes,_0_no_ 
                              "Clinical_data_Fatty_liver_in_ultrasound_1_yes,_0_no_" 
                                               Clinical_data_Diabetes__1_yes,_0__no_ 
                                             "Clinical_data_Diabetes__1_yes,_0__no_" 
                                                             Clinical_data_Hba1C__%_ 
                                                           "Clinical_data_Hba1C__%_" 
                                            Clinical_data_Hypertension__1_yes,_0_no_ 
                                          "Clinical_data_Hypertension__1_yes,_0_no_" 
                                                    Clinical_data_MetS__1_yes,_0_no_ 
                                                  "Clinical_data_MetS__1_yes,_0_no_" 
                                     Clinical_data_TNM_-_Classification_of_carcinoma 
                                   "Clinical_data_TNM_-_Classification_of_carcinoma" 
                                                             Clinical_data_Histology 
                                                           "Clinical_data_Histology" 
                             Clinical_data_Localization_in_colon__right_left_rectum_ 
                           "Clinical_data_Localization_in_colon__right_left_rectum_" 
                                                                 Fast_food_Every_day 
                                                               "Fast_food_Every_day" 
                                                                Fast_food_Every_week 
                                                              "Fast_food_Every_week" 
                                             Fast_food_Less_than_every_week_or_never 
                                           "Fast_food_Less_than_every_week_or_never" 
                                 Meat_in_g_week_Red_meat__pork,_beef,_veal,_venison_ 
                               "Meat_in_g_week_Red_meat__pork,_beef,_veal,_venison_" 
                                         Meat_in_g_week_White_meat__chicken,_turkey_ 
                                       "Meat_in_g_week_White_meat__chicken,_turkey_" 
                          Meat_in_g_week_Total__all_meat,_i.e._red,_white_and_other_ 
                        "Meat_in_g_week_Total__all_meat,_i.e._red,_white_and_other_" 
                                                                         Fish_g_week 
                                                                       "Fish_g_week" 
                                                                   Vegetables_g_week 
                                                                 "Vegetables_g_week" 
                                                                       Fruits_g_week 
                                                                     "Fruits_g_week" 
                                                             Smoking_0_never,_1_ever 
                                                           "Smoking_0_never,_1_ever" 
                                                                     Smoking_Current 
                                                                   "Smoking_Current" 
                                          Smoking_1_more_than_or_equal_to_1_pack_day 
                                        "Smoking_1_more_than_or_equal_to_1_pack_day" 
                                          Smoking_1_more_than_or_equal_to_2_pack_day 
                                        "Smoking_1_more_than_or_equal_to_2_pack_day" 
                                                             Physical_activity_1_low 
                                                           "Physical_activity_1_low" 
                                                    Physical_activity_1_intermediate 
                                                  "Physical_activity_1_intermediate" 
                                                            Physical_activity_1_high 
                                                          "Physical_activity_1_high" 
  Physical_activity_1_physical_activity_available,_0_physical_activity_not_available 
"Physical_activity_1_physical_activity_available,_0_physical_activity_not_available" 
                                        MLG_classifier_Type_Probability_of_carcinoma 
                                      "MLG_classifier_Type_Probability_of_carcinoma" 
                                 MLG_classifier_Type_Probability_of_advanced_adenoma 
                               "MLG_classifier_Type_Probability_of_advanced_adenoma" 

After having created vectros with either continous or factorial data (just name holders), and produced the desired scatter plot funciton, I can test the function outside a loop, unsing two arbitrary continous variables

# Try functions outside a loop
scatter_fun(data = mdFengQ_2015_SupplData1_4,
           x = "Clinical_data_GPT__ALT___U_L_", y = "Clinical_data_CRP__mg_L_")

Then loop through all continous variables and cross-analyze all of them, and Show a single plot from the collection

# looping through only Scatter Plot data (continuous)
all_plots = map(numericVar, function(resp) {
  map(numericVar, function(i) {
    scatter_fun(data = mdFengQ_2015_SupplData1_4, x = i, y = resp)
  })
})

# print single plot
all_plots$Clinical_data_Age__yrs_$Clinical_data_GGT__U_L_

ProduceGroup plots based on the same variables as a y-axis

colleciton_plots = map(all_plots, ~cowplot::plot_grid(plotlist = .x))
colleciton_plots$Clinical_data_Age__yrs_

colleciton_plots
$Sample_ID

$Clinical_data_Age__yrs_

$Clinical_data_BMI

$Clinical_data_Waist__cm_

$Clinical_data_Hip__cm_

$Clinical_data_GGT__U_L_

$Clinical_data_GOT__AST___U_L_

$Clinical_data_GPT__ALT___U_L_

$Clinical_data_Fasting_insulin__µU_mL_

$Clinical_data_Fasting_glucose__mg_L_

$Clinical_data_HOMA_index

$Clinical_data_CRP__mg_L_

$Clinical_data_Ferritin__ng_mL_

$Clinical_data_Hb__g_L_

$Clinical_data_TG__mg_L_

$Clinical_data_HDL__mg_L_

$Clinical_data_LDL__mg_L_

$Fiber_intake_g_week

$MLG_classifier_Probability_of_carcinoma

$MLG_classifier_Probability_of_advanced_adenoma

I can now proceed to produce visuals for factorial data (x-asis) vs continuos data (y-axis) using violin plots

## for facterial data, use violin
violin_fun = function(data,x, y) {
  ggplot(data, aes(x = .data[[x]], y = .data[[y]]) ) +
    geom_violin() +
    geom_jitter(size = 1, alpha = 0.5, width = 0.2) +
    theme_bw() +
    theme(axis.text = element_blank(),axis.title = element_text(size = 5))
}

# Try functions outside a loop
violin_fun(data = mdFengQ_2015_SupplData1_4,
           x = "Clinical_data_State", y = "Clinical_data_BMI")

Loop through

# looping through only Violin Plot data (cont vs Factorial)
all_plots2 = map(factorialVar, function(resp) {
  map(numericVar, function(i) {
    violin_fun(data = mdFengQ_2015_SupplData1_4, y = i, x = resp)
  })
})
colleciton_plots2 = map(all_plots2, ~cowplot::plot_grid(plotlist = .x))

And print

colleciton_plots2
$Clinical_data_State

$`Clinical_data_WHR__waist-hip-ratio_`

$`Clinical_data_Gender__1_male,_2_female_`

$`Clinical_data_Fatty_liver_in_ultrasound_1_yes,_0_no_`

$`Clinical_data_Diabetes__1_yes,_0__no_`

$`Clinical_data_Hba1C__%_`

$`Clinical_data_Hypertension__1_yes,_0_no_`

$`Clinical_data_MetS__1_yes,_0_no_`

$`Clinical_data_TNM_-_Classification_of_carcinoma`

$Clinical_data_Histology

$Clinical_data_Localization_in_colon__right_left_rectum_

$Fast_food_Every_day

$Fast_food_Every_week

$Fast_food_Less_than_every_week_or_never

$`Meat_in_g_week_Red_meat__pork,_beef,_veal,_venison_`

$`Meat_in_g_week_White_meat__chicken,_turkey_`

$`Meat_in_g_week_Total__all_meat,_i.e._red,_white_and_other_`

$Fish_g_week

$Vegetables_g_week

$Fruits_g_week

$`Smoking_0_never,_1_ever`

$Smoking_Current

$Smoking_1_more_than_or_equal_to_1_pack_day

$Smoking_1_more_than_or_equal_to_2_pack_day

$Physical_activity_1_low

$Physical_activity_1_intermediate

$Physical_activity_1_high

$`Physical_activity_1_physical_activity_available,_0_physical_activity_not_available`

$MLG_classifier_Type_Probability_of_carcinoma

$MLG_classifier_Type_Probability_of_advanced_adenoma

LS0tCnRpdGxlOiAiY3VyYXRlZE1ldGFnZW5vbWljIG1ldGFkYXRhIgpvdXRwdXQ6CiAgaHRtbF9kb2N1bWVudDoKICAgIGRmX3ByaW50OiBwYWdlZAotLS0KCiMgRGF0YSBFeHBsb3JhdGlvbgp1c2UgY3VyYXRlZE1ldGFnZW5vbWljRGF0YS50c3YgdG8gZXhwbG9yZSBkYXRhCgpgYGB7cn0KbGlicmFyeSh0aWR5dmVyc2UpCmxpYnJhcnkocmVhZHhsKQpkZiA8LSByZWFkX3RzdigicmF3LWRhdGEvY3VyYXRlZE1ldGFnZW5vbWljRGF0YS50c3YiKQoKZGYgJT4lCiAgZ3JvdXBfYnkoc3R1ZHlfbmFtZSxzdHVkeV9jb25kaXRpb24pICU+JQogIHN1bW1hcmlzZShjb3VudCA9IG4oKSkKCmBgYApmaW5kIHN0dWRpZXMgdGhhdCBpbmNsdWRlZCBhbiBpbnRlcm5hbCBjb250cm9sCgpgYGB7ciBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQpkZjEgPC0gZGYgJT4lIAogIGdyb3VwX2J5KHN0dWR5X25hbWUsc3R1ZHlfY29uZGl0aW9uKSAlPiUKICBzdW1tYXJpc2UoY291bnQgPSBuKCkpICU+JQogIHNlbGVjdChzdHVkeV9uYW1lKSAlPiUgYXMubGlzdCgpCnZlYyA8LSBkZjEkc3R1ZHlfbmFtZQpzdHVkeU5hbWVzV2l0aEN0ciA8LSB2ZWNbYXZlKHNlcV9hbG9uZyh2ZWMpLCB2ZWMsIEZVTiA9IGxlbmd0aCk+MV0gJT4lIHVuaXF1ZSgpCnN0dWR5TmFtZXNXaXRoQ3RyCmBgYApub3cgdGhhdCBJIGhhdmUgdGhlIGxpc3Qgb2Ygc3R1ZGllcyB3aXRoIGNvbnRyb2xzLCBJJ2xsIGNoZWNrIHRoZSBmaXJzdCB0byBtYWtlIHN1cmUgbXkgc2VhcmNoIGlzIGFjY3VyYXRlCmBgYHtyIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9CmRmICU+JQogIGdyb3VwX2J5KHN0dWR5X25hbWUsc3R1ZHlfY29uZGl0aW9uKSAlPiUKICBzdW1tYXJpc2UoY291bnQgPSBuKCkpICU+JQogIGZpbHRlcihzdHVkeV9uYW1lID09IHN0dWR5TmFtZXNXaXRoQ3RyWzFdKQpgYGAKdGhlIGxpc3Qgc2VlbXMgdG8gaGF2ZSB3b3JrZWQhIE5vdyBmaW5kIENSQyBzdHVkaWVzIHRoYXQgaW5jbHVkZSBpbnRlcm5hbCBDdHJzCmBgYHtyIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9CnN0dWRpZXNDUkN3aXRoQ3RyIDwtIGRmICU+JQogIGdyb3VwX2J5KHN0dWR5X25hbWUsc3R1ZHlfY29uZGl0aW9uKSAlPiUKICBzdW1tYXJpc2UoY291bnQgPSBuKCkpICU+JQogIGZpbHRlcihzdHVkeV9uYW1lICVpbiUgc3R1ZHlOYW1lc1dpdGhDdHJbMTpsZW5ndGgoc3R1ZHlOYW1lc1dpdGhDdHIpXSkgJT4lCiAgZmlsdGVyKHN0dWR5X2NvbmRpdGlvbiA9PSAiQ1JDIikKc3R1ZGllc0NSQ3dpdGhDdHIKYGBgClNhdmUgdGFibGUKCmBgYHtyfQp3cml0ZV9jc3Yoc3R1ZGllc0NSQ3dpdGhDdHIsImN1cmF0ZWQtZGF0YS9zdHVkaWVzQ1JDd2l0aEN0ci5jc3YiKQpgYGAKCgpub3cgdGhhdCBJIGhhdmUgYSB0YWJsZSB3aXRoIG9ubHkgQ1JDIHN0dWRpZXMgd2l0aCBpbmNsdWRlZCBDdHJzLCBJJ2xsIGxvb2sgYXQgdGhlaXIgTWV0YWRhdGEKYGBge3IgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0Kc3R1ZGllc0NSQ3dpdGhDdHIkc3R1ZHlfbmFtZVsxXQpkZiAlPiUKICBmaWx0ZXIoc3R1ZHlfbmFtZSA9PSBzdHVkaWVzQ1JDd2l0aEN0ciRzdHVkeV9uYW1lWzFdKQpgYGAKIyBGZW5nUV8yMDE1IApUaGlzIHN0dWR5IGlzIHRoZSBmaXJzdCBzdHVkeSBmcm9tIGxpc3Qgd2l0aCBDUkMgYW5kIGludGVybmFsIEN0cnMuIEFsdGhvdWdoIHRoZXJlIGlzIG5vIGFubm90YXRpb24gaW4gdGhlIGN1cmF0ZWRNZXRhZ2Vub21pYyB0YWJsZSwgdGhlIG9yaWdpbmFsIGFydGljbGUgc2VlbSB0byBoYXZlIHNvbWUgSSBzZWUgaWYgSSBjYW4gcHVsbCB0aGUgdGFibGUgYW5kIG1lcmdlLiBCZWxvdyBpcyBhIGJ1bmNoIG9mIGNvZGUgdGhhdCBjbGVhbiB1cCBDbGluaWNhbCB0YWJsZSBkb3dubG9hZWQgZnJvbSBvcmlnaW5hbCBtYW51c2NyaXB0CgpgYGB7ciBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQojIEkgY29uZmlybWVkIHRoZXJlIGlzIGxvdHMgb2YgZGF0YSBpbiB0aGUgcHVibGlzaGVkIGRhdGFzZXQKIyBjdXJhdGUgdGhlIHRhYmxlCgptZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV8xIDwtIHJlYWRfZXhjZWwoInJhdy1kYXRhL0ZlbmdRXzIwMTVfUE1JRDI1NzU4NjQyLzQxNDY3XzIwMTVfQkZuY29tbXM3NTI4X01PRVNNMTE5M19FU00ueGxzeCIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBza2lwID0gMikKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIApuZXdIZWFkZXIgPC0gc3RyX3JlcGxhY2Uoc3RyaW5nID0gbmFtZXMobWRGZW5nUV8yMDE1X1N1cHBsRGF0YTFfMSkscGF0dGVybiA9ICJcXC5cXC5cXC4uKyIscmVwbGFjZW1lbnQgPSAiIikKCmZvciAoaSBpbiAxOmxlbmd0aChuZXdIZWFkZXIpKXsKICBob2xkIDwtIG5ld0hlYWRlcltpXQogIGlmIChuZXdIZWFkZXJbaSsxXSA9PSAiIil7CiAgICBuZXdIZWFkZXJbaSsxXSA8LSBob2xkCiAgfQogICMgYnJlYWsgY2hlY2sKICBpZiAoaT09NTEpIGJyZWFrCn0KCm5hbWVzKG1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzEpIDwtIHBhc3RlMChuZXdIZWFkZXIsIl8iLG1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzFbMSxdKQptZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV8yIDwtIG1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzFbLTEsXQoKbmFtZXMobWRGZW5nUV8yMDE1X1N1cHBsRGF0YTFfMilbNDhdIDwtICJNTEcgY2xhc3NpZmllcl9UeXBlX1Byb2JhYmlsaXR5IG9mIGNhcmNpbm9tYSIKbmFtZXMobWRGZW5nUV8yMDE1X1N1cHBsRGF0YTFfMilbNTBdIDwtICJNTEcgY2xhc3NpZmllcl9UeXBlX1Byb2JhYmlsaXR5IG9mIGFkdmFuY2VkIGFkZW5vbWEiCm5hbWVzKG1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzIpWzUyXSA8LSAiTUxHIGNsYXNzaWZpZXJfVHlwZV9Qcm9iYWJpbGl0eSBvZiBhZHZhbmNlZCBhZGVub21hIGFkZCBhZ2UiCm5hbWVzKG1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzIpWzFdIDwtICJTYW1wbGUgSUQiCgpuYW1lcyhtZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV8yKSA8LSBzdHJfcmVwbGFjZV9hbGwoc3RyaW5nID0gbmFtZXMobWRGZW5nUV8yMDE1X1N1cHBsRGF0YTFfMikscGF0dGVybiA9ICJbWzpzcGFjZTpdXXxcXCh8XFwvfFxcKXxcXDoiLHJlcGxhY2VtZW50ID0gIl8iKQoKbWRGZW5nUV8yMDE1X1N1cHBsRGF0YTFfMyA8LSBtZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV8yICU+JQogIG11dGF0ZShhY3Jvc3MobmFtZXMobWRGZW5nUV8yMDE1X1N1cHBsRGF0YTFfMlssMzoyNV0pLCBhcy5pbnRlZ2VyKSkgJT4lCiAgbXV0YXRlKGFjcm9zcyhuYW1lcyhtZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV8yWywyNjozMV0pLCBhcy5mYWN0b3IpKSAlPiUKICBtdXRhdGUoYWNyb3NzKG5hbWVzKG1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzJbLDMyOjM4XSksIGFzLmludGVnZXIpKSAlPiUKICBtdXRhdGUoYWNyb3NzKG5hbWVzKG1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzJbLDM5OjQ2XSksIGFzLmZhY3RvcikpICU+JQogIG11dGF0ZShhY3Jvc3MobmFtZXMobWRGZW5nUV8yMDE1X1N1cHBsRGF0YTFfMlssNDddKSwgYXMubnVtZXJpYykpICU+JQogIG11dGF0ZShhY3Jvc3MobmFtZXMobWRGZW5nUV8yMDE1X1N1cHBsRGF0YTFfMlssNDhdKSwgYXMuZmFjdG9yKSkgJT4lCiAgbXV0YXRlKGFjcm9zcyhuYW1lcyhtZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV8yWyw0OV0pLCBhcy5udW1lcmljKSkgJT4lCiAgbXV0YXRlKGFjcm9zcyhuYW1lcyhtZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV8yWyw0MF0pLCBhcy5mYWN0b3IpKQoKbWRGZW5nUV8yMDE1X1N1cHBsRGF0YTFfMyA8LSBtZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV8zICU+JQogIG11dGF0ZShDbGluaWNhbF9kYXRhX1N0YXRlID0gZmFjdG9yKENsaW5pY2FsX2RhdGFfU3RhdGUsIGxldmVscz1jKCJjb250cm9scyIsICJhZHZhbmNlZCBhZGVub21hIiwiY2FyY2lub21hIikpKQoKI2VkaXQgdGFibGVzIGFzIHNvbWUgbXViZXJpYyBhcmUgYWN0dWFsbHkgZmFjdG9ycwptZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV80IDwtIG1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzMgJT4lCiAgbXV0YXRlKGFjcm9zcyhuYW1lcyhtZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV8zWyw3OjldKSwgYXMuZmFjdG9yKSkgJT4lCiAgbXV0YXRlKGFjcm9zcyhuYW1lcyhtZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV8zWywxNjoxN10pLCBhcy5mYWN0b3IpKSAlPiUKICBtdXRhdGUoYWNyb3NzKG5hbWVzKG1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzNbLDI0OjM3XSksIGFzLmZhY3RvcikpICU+JQogIG11dGF0ZShhY3Jvc3MobmFtZXMobWRGZW5nUV8yMDE1X1N1cHBsRGF0YTFfM1ssMzk6NDZdKSwgYXMuZmFjdG9yKSkgJT4lCiAgbXV0YXRlKGFjcm9zcyhuYW1lcyhtZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV8zWyw0OF0pLCBhcy5mYWN0b3IpKSAlPiUKICBtdXRhdGUoYWNyb3NzKG5hbWVzKG1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzNbLDUwXSksIGFzLmZhY3RvcikpCgoKIyBJIGhhdmUgbm93IGEgd2VsbCBmb3JtYXR0ZWQgdGFibGUKCm1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzQKYGBgCgpJIGNhbiBub3cgIHZpZHVhbGl6ZSBzb21lIGRhdGEgZm9yIGV4cGxvcmF0aW9uIHB1cnBvc2UKCmBgYHtyIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9CgojIEFnZSBhY3Jvc3MgU3RhdHVzCm1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzMgJT4lCiAgbXV0YXRlKENsaW5pY2FsX2RhdGFfU3RhdGUgPSBmYWN0b3IoQ2xpbmljYWxfZGF0YV9TdGF0ZSwgbGV2ZWxzPWMoImNvbnRyb2xzIiwgImFkdmFuY2VkIGFkZW5vbWEiLCJjYXJjaW5vbWEiKSkpICU+JQogIAogIGdncGxvdChhZXMoeCA9IENsaW5pY2FsX2RhdGFfU3RhdGUseSA9IGBDbGluaWNhbF9kYXRhX0FnZV8oeXJzKWApKSArCiAgZ2VvbV92aW9saW4odHJpbSA9IFQsbmEucm0gPSBUKSArCiAgZ2VvbV9qaXR0ZXIod2lkdGggPSAwLjIsc2l6ZSA9IDEpICsKICAKICB0aGVtZV9jbGFzc2ljKCkKCiMgQk1JIGFjcm9zcyBTdGF0dXMKbWRGZW5nUV8yMDE1X1N1cHBsRGF0YTFfMyAlPiUKICBtdXRhdGUoQ2xpbmljYWxfZGF0YV9TdGF0ZSA9IGZhY3RvcihDbGluaWNhbF9kYXRhX1N0YXRlLCBsZXZlbHM9YygiY29udHJvbHMiLCAiYWR2YW5jZWQgYWRlbm9tYSIsImNhcmNpbm9tYSIpKSkgJT4lCiAgCiAgZ2dwbG90KGFlcyh4ID0gQ2xpbmljYWxfZGF0YV9TdGF0ZSx5ID0gQ2xpbmljYWxfZGF0YV9CTUkpKSArCiAgZ2VvbV92aW9saW4odHJpbSA9IFQsbmEucm0gPSBUKSArCiAgZ2VvbV9qaXR0ZXIod2lkdGggPSAwLjIsc2l6ZSA9IDEpICsKICAKICB0aGVtZV9jbGFzc2ljKCkKCiMgU2V4IGFjcm9zcyBTdGF0dXMKbWRGZW5nUV8yMDE1X1N1cHBsRGF0YTFfMyAlPiUKICBtdXRhdGUoQ2xpbmljYWxfZGF0YV9TdGF0ZSA9IGZhY3RvcihDbGluaWNhbF9kYXRhX1N0YXRlLCBsZXZlbHM9YygiY29udHJvbHMiLCAiYWR2YW5jZWQgYWRlbm9tYSIsImNhcmNpbm9tYSIpKSkgJT4lCiAgZ3JvdXBfYnkoQ2xpbmljYWxfZGF0YV9TdGF0ZSxgQ2xpbmljYWxfZGF0YV9HZW5kZXJfKDE6bWFsZSxfMjpmZW1hbGUpYCkgJT4lCiAgc3VtbWFyaXNlKGNvdW50ID0gbigpKQoKIyBHR1QgYWNyb3NzIFN0YXR1cwptZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV8zICU+JQogIG11dGF0ZShDbGluaWNhbF9kYXRhX1N0YXRlID0gZmFjdG9yKENsaW5pY2FsX2RhdGFfU3RhdGUsIGxldmVscz1jKCJjb250cm9scyIsICJhZHZhbmNlZCBhZGVub21hIiwiY2FyY2lub21hIikpKSAlPiUKICAKICBnZ3Bsb3QoYWVzKHggPSBDbGluaWNhbF9kYXRhX1N0YXRlLHkgPSBgQ2xpbmljYWxfZGF0YV9HR1RfKFUvTClgKSkgKwogIGdlb21fdmlvbGluKHRyaW0gPSBULG5hLnJtID0gVCkgKwogIGdlb21faml0dGVyKHdpZHRoID0gMC4yLHNpemUgPSAxKSArCiAgCiAgdGhlbWVfY2xhc3NpYygpCgpgYGAKCiMjIEZhc3RlciBDb3JyZWxhdGlvbiBWaXN1YWxpemF0aW9uClBhcnQgb2YgdGhlIGZ1dHVyZSB3b3JrIHdpbGwgaW5jbHVkZSB0aGUgaWRlbnRpZmljYXRpb25vZiBjb3JyZWxhdGl2ZSB2YWx1ZXMgYmV0d2VlbiBjZXJ0YWluIFRheGEgb3IgQkdDIGFuZCBtZXRhZGF0YS4gVG8gc3BlZWQgdXAgcmVwcmVzZW50YXRpb24sIEkgZGl2aWRlIHRoZSB2YXJpYWJsZXMgKGNvbHVtbnMpIGJldHdlZW4gY29udGlub3VzIGFuZCBmYWN0b3JpYWwsIGFuZCBhZG9wdCBkaWZmZXJuZXQgcGxvdHRpbmcgc3RyYXRlZ3kgdG8gaWRlbnRpZnkgbGluZWFyIHJlZ3Jlc3Npb24KCmBgYHtyfQojIExvb3AgdG8gcHJpbnQgbXVsdGlwbGUgZXhwbG9yYXRvcnkgUGxvdHMKIyMgY2xhc3NpZnkgdmFyaWFibGVzIGFyZSBudW1lcmljIG9yIGZhY3RvcmlhbAoKbnVtZXJpY1ZhciA8LSBtZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV80ICU+JQogIHNlbGVjdF9pZigucHJlZGljYXRlID0gaXMubnVtZXJpYykgJT4lIAogIG5hbWVzKCkKCmZhY3RvcmlhbFZhciA8LSBtZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV80ICU+JQogIHNlbGVjdF9pZigucHJlZGljYXRlID0gaXMuZmFjdG9yKSAlPiUgCiAgbmFtZXMoKQoKI2dpdmUgbmFtZSB0byBlYWNoIHZhbHVlIGluIHRoZSB2ZWN0b3JzCm51bWVyaWNWYXIgPSBzZXRfbmFtZXMobnVtZXJpY1ZhcikKZmFjdG9yaWFsVmFyID0gc2V0X25hbWVzKGZhY3RvcmlhbFZhcikKCiMgbWFrZSBwbG90dGluZyBmdW5jdGlvbnMKIyMgZm9yIGNvbnRpbnVvdXMgZGF0YSwgdXNlIHNjYXR0ZXIKc2NhdHRlcl9mdW4gPSBmdW5jdGlvbihkYXRhLHgseSl7CiAgZ2dwbG90KGRhdGEsIGFlcyh4ID0gLmRhdGFbW3hdXSwgeSA9IC5kYXRhW1t5XV0pKSArCiAgICBnZW9tX3BvaW50KG5hLnJtPVRSVUUpICsKICAgIGdlb21fc21vb3RoKG1ldGhvZD0nbG0nLG5hLnJtPVRSVUUpICsKICAgIHRoZW1lX2J3KCkgKwogICAgdGhlbWUoYXhpcy50ZXh0ID0gZWxlbWVudF9ibGFuaygpLGF4aXMudGl0bGUgPSBlbGVtZW50X3RleHQoc2l6ZSA9IDUpKQp9CmBgYAoKQWZ0ZXIgaGF2aW5nIGNyZWF0ZWQgdmVjdHJvcyB3aXRoIGVpdGhlciBjb250aW5vdXMgb3IgZmFjdG9yaWFsIGRhdGEgKGp1c3QgbmFtZSBob2xkZXJzKSwgYW5kIHByb2R1Y2VkIHRoZSBkZXNpcmVkIHNjYXR0ZXIgcGxvdCBmdW5jaXRvbiwgSSBjYW4gdGVzdCB0aGUgZnVuY3Rpb24gb3V0c2lkZSBhIGxvb3AsIHVuc2luZyB0d28gYXJiaXRyYXJ5IGNvbnRpbm91cyB2YXJpYWJsZXMgCgpgYGB7ciBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQojIFRyeSBmdW5jdGlvbnMgb3V0c2lkZSBhIGxvb3AKc2NhdHRlcl9mdW4oZGF0YSA9IG1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzQsCiAgICAgICAgICAgeCA9ICJDbGluaWNhbF9kYXRhX0dQVF9fQUxUX19fVV9MXyIsIHkgPSAiQ2xpbmljYWxfZGF0YV9DUlBfX21nX0xfIikKYGBgClRoZW4gbG9vcCB0aHJvdWdoIGFsbCBjb250aW5vdXMgdmFyaWFibGVzIGFuZCBjcm9zcy1hbmFseXplIGFsbCBvZiB0aGVtLCBhbmQgU2hvdyBhIHNpbmdsZSBwbG90IGZyb20gdGhlIGNvbGxlY3Rpb24KCmBgYHtyIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9CiMgbG9vcGluZyB0aHJvdWdoIG9ubHkgU2NhdHRlciBQbG90IGRhdGEgKGNvbnRpbnVvdXMpCmFsbF9wbG90cyA9IG1hcChudW1lcmljVmFyLCBmdW5jdGlvbihyZXNwKSB7CiAgbWFwKG51bWVyaWNWYXIsIGZ1bmN0aW9uKGkpIHsKICAgIHNjYXR0ZXJfZnVuKGRhdGEgPSBtZEZlbmdRXzIwMTVfU3VwcGxEYXRhMV80LCB4ID0gaSwgeSA9IHJlc3ApCiAgfSkKfSkKCiMgcHJpbnQgc2luZ2xlIHBsb3QKYWxsX3Bsb3RzJENsaW5pY2FsX2RhdGFfQWdlX195cnNfJENsaW5pY2FsX2RhdGFfR0dUX19VX0xfCmBgYAoKUHJvZHVjZUdyb3VwIHBsb3RzIGJhc2VkIG9uIHRoZSBzYW1lIHZhcmlhYmxlcyBhcyBhIHktYXhpcwoKYGBge3IgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0KY29sbGVjaXRvbl9wbG90cyA9IG1hcChhbGxfcGxvdHMsIH5jb3dwbG90OjpwbG90X2dyaWQocGxvdGxpc3QgPSAueCkpCmBgYAoKYGBge3IgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0KY29sbGVjaXRvbl9wbG90cwpgYGAKCkkgY2FuIG5vdyBwcm9jZWVkIHRvIHByb2R1Y2UgdmlzdWFscyBmb3IgZmFjdG9yaWFsIGRhdGEgKHgtYXNpcykgdnMgY29udGludW9zIGRhdGEgKHktYXhpcykgdXNpbmcgdmlvbGluIHBsb3RzCgpgYGB7cn0KIyMgZm9yIGZhY3RlcmlhbCBkYXRhLCB1c2UgdmlvbGluCnZpb2xpbl9mdW4gPSBmdW5jdGlvbihkYXRhLHgsIHkpIHsKICBnZ3Bsb3QoZGF0YSwgYWVzKHggPSAuZGF0YVtbeF1dLCB5ID0gLmRhdGFbW3ldXSkgKSArCiAgICBnZW9tX3Zpb2xpbigpICsKICAgIGdlb21faml0dGVyKHNpemUgPSAxLCBhbHBoYSA9IDAuNSwgd2lkdGggPSAwLjIpICsKICAgIHRoZW1lX2J3KCkgKwogICAgdGhlbWUoYXhpcy50ZXh0ID0gZWxlbWVudF9ibGFuaygpLGF4aXMudGl0bGUgPSBlbGVtZW50X3RleHQoc2l6ZSA9IDUpKQp9CgojIFRyeSBmdW5jdGlvbnMgb3V0c2lkZSBhIGxvb3AKdmlvbGluX2Z1bihkYXRhID0gbWRGZW5nUV8yMDE1X1N1cHBsRGF0YTFfNCwKICAgICAgICAgICB4ID0gIkNsaW5pY2FsX2RhdGFfU3RhdGUiLCB5ID0gIkNsaW5pY2FsX2RhdGFfQk1JIikKYGBgCgpMb29wIHRocm91Z2gKCmBgYHtyIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9CiMgbG9vcGluZyB0aHJvdWdoIG9ubHkgVmlvbGluIFBsb3QgZGF0YSAoY29udCB2cyBGYWN0b3JpYWwpCmFsbF9wbG90czIgPSBtYXAoZmFjdG9yaWFsVmFyLCBmdW5jdGlvbihyZXNwKSB7CiAgbWFwKG51bWVyaWNWYXIsIGZ1bmN0aW9uKGkpIHsKICAgIHZpb2xpbl9mdW4oZGF0YSA9IG1kRmVuZ1FfMjAxNV9TdXBwbERhdGExXzQsIHkgPSBpLCB4ID0gcmVzcCkKICB9KQp9KQpjb2xsZWNpdG9uX3Bsb3RzMiA9IG1hcChhbGxfcGxvdHMyLCB+Y293cGxvdDo6cGxvdF9ncmlkKHBsb3RsaXN0ID0gLngpKQpgYGAKCkFuZCBwcmludApgYGB7ciBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQpjb2xsZWNpdG9uX3Bsb3RzMgpgYGAKCgoKCgo=